{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import csv\n",
    "import time\n",
    "def data(page):\n",
    "    s = requests.session()\n",
    "    url = 'https://www.lagou.com/jobs/positionAjax.json?city=%E4%B8%8A%E6%B5%B7&needAddtionalResult=false'\n",
    "    headers = {\n",
    "            'Accept': \"application/json, text/javascript, */*; q=0.01\",\n",
    "            'User-Agent': \"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36\",\n",
    "            'Referer':'https://www.lagou.com/jobs/list_%E8%BD%AF%E4%BB%B6%E6%B5%8B%E8%AF%95%E5%B7%A5%E7%A8%8B%E5%B8%88?city=%E4%B8%8A%E6%B5%B7&cl=false&fromSearch=true&labelWords=&suginput='\n",
    "    }\n",
    "    form_data = {\n",
    "            'first': 'true',\n",
    "            'pn': page,\n",
    "            'kd': '软件测试工程师'\n",
    "    }\n",
    "    url_list = 'https://www.lagou.com/jobs/list_%E8%BD%AF%E4%BB%B6%E6%B5%8B%E8%AF%95%E5%B7%A5%E7%A8%8B%E5%B8%88?city=%E4%B8%8A%E6%B5%B7&cl=false&fromSearch=true&labelWords=&suginput='\n",
    "    s.get(url_list, headers=headers,timeout=3)\n",
    "    cookie = s.cookies\n",
    "    response = s.post(url, data=form_data, headers=headers,cookies=cookie,timeout=3)\n",
    "    response.raise_for_status()\n",
    "    response.encoding = response.apparent_encoding\n",
    "    job_json = response.json()\n",
    "    job_list = job_json['content']['positionResult']['result']\n",
    "    csv_data = []\n",
    "    for i in job_list:\n",
    "        job_info = []\n",
    "        job_info.append(i['positionName'])  \n",
    "        job_info.append(i['companyShortName'])  \n",
    "        job_info.append(i['salary'])    \n",
    "        job_info.append(i['education'])  \n",
    "        job_info.append(i['district'])  \n",
    "        job_info.append(i['workYear'])  \n",
    "        job_info.append(i['positionAdvantage'])  \n",
    "        csv_data.append(job_info)\n",
    "    print(csv_data)\n",
    "    csvfile = open('软件职业.csv', 'a+',encoding='utf-8-sig',newline='')\n",
    "    writer = csv.writer(csvfile)\n",
    "    writer.writerows(csv_data)\n",
    "    csvfile.close()\n",
    "    return csv_data\n",
    "if __name__ == '__main__':\n",
    "    a = [('职位','公司','薪资','学历','位置','工作经验要求','福利待遇')]\n",
    "    csvfile = open('软件职业.csv', 'a+',encoding='utf-8-sig',newline='')\n",
    "    writer = csv.writer(csvfile)\n",
    "    writer.writerows(a)\n",
    "    csvfile.close()\n",
    "    all_company = []\n",
    "    for page_num in range(1, 100):\n",
    "        result = data(page=page_num)\n",
    "        all_company += result\n",
    "        print('已抓取{}页, 总职位数:{}'.format(page_num, len(all_company)))\n",
    "        time.sleep(15)  \n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
